<?php
$addreturn = array();
$oldurl="";
$oldnextpage="";
$TTBB=array();
include_once(APP_PATH."/collect/config.php");
include_once(APP_PATH . "/collect/Lib/CollcetLib/seturls.class.php");
require_once(APP_PATH . "/collect/Common/common.php");
require_once(APP_PATH . "/collect/Common/yzs1013_pldr.php");
require_once(APP_PATH . "/collect/Lib/ORG/Zqindex.class.php");
require_once(APP_PATH . "/collect/Lib/ORG/Zqnews.class.php");

class CollectTongyong
{
    public $url,$nn55;
    public $gather_type;
    public $rs;

    public function getPageContent($url, $gather_type = 1,$bug=false,$zqnews=null,$title="",$dateline="",$pagehtml="")
    {
        global $bbsdata,$oldnextpage,$htmlpageuse_first;
        $this->url=$url;
        $this->gather_type=$gather_type;

        $yurl=$url;
        if(preg_match("/[\r\n\;]+(?=http)/",$url)){
            $nn888=preg_split("/[\r\n;]+/",$url);
            $url=$nn888[0];
        }

        if($zqnews==null){
            //内容 步骤2
            $news = new Zqnews($url,$pagehtml,$bug,$gather_type,$title,$dateline);
        }else{
            $news=$zqnews;
            $news->seturl($url);
            if($news->isfail) return null;
        }

        $this->rs=$news->rs;
        if($news->yuannr=="kongHTML"){
            return "kongHTML";
        }
        //刚抓了一篇下来，看看有没有分页哦~
        if(empty($news->nn888)&&!empty($nn888)){
            $news->nn888=$nn888;
        }elseif(!empty($news->nn888)){
            $nn888=$news->nn888;
        }

        //多线程下载分页内容
        if(!empty($nn888)&&is_array($nn888)){
            array_shift($nn888); //弹出第一个元素
            $newarr=array();
            for($i=0;$i<count($nn888);$i++){
                $id=$news->t->sqlvalue("select id from `cache` where url='".$nn888[$i]."'");
                if($id<=1||$id==0){
                    $newarr[]=$nn888[$i];
                }
            }
            $contents=multiDownload($newarr);
            unset($newarr);
            if(!empty($contents)) {
                foreach ($contents as $url => $content) {
                    $news->saveCacheTempnr($url, $content);
                }
            }
        }
        $news->gather_type=$gather_type;
        $news->nozqnr=($_REQUEST['nozqnr'])?$_REQUEST['nozqnr']:0;

        if($news->pageinfo==""||$news->pageinfo==null) {
            //内容 步骤5 获取内容各项数据 ==getdata
            $pageinfo = $news->getdata();
        }else{
            if(is_string($news->pageinfo)){
                $data=json_decode($news->pageinfo,1);
            }else{
                $data=$news->pageinfo;
            }
            if(is_string($data)){
                $news->pageinfo=$data;
                $data=json_decode($data,1);
            }
            if(is_string($data)){
                $news->pageinfo=$data;
                $data=json_decode($data,1);
            }
            if(is_array($data)&&($data['message']!=""||$data['flash']!=""||!empty($data['prodata']))){
                $pageinfo =$data;
            }
        }

        if($pageinfo=="islist"&&($news->flash==""&&$news->swfstr=="")||$pageinfo==null){
            return "islist";
        }
        $havenofinish=false;
        if(!empty($news->nn888)) {
            for ($i = 0; $i < count($news->nn888); $i++) {
                $id = $news->t->sqlvalue("select id from `cache` where url='" . $news->nn888[$i] . "' and (message is null or message='')");
                if ($id > 0) {
                    $havenofinish = true;
                    break;
                }
            }
        }
        if($news->isfromcache&&!$havenofinish){
            if(is_string($pageinfo)){
                $pageinfo=json_decode($pageinfo,1);
            }
            return $pageinfo;
        }

        $pageinfo['subject']=(trim($news->title)!=""&&trim($news->title)!="（无标题）"&&$news->title!=$pageinfo['subject'])?$news->title:$pageinfo['subject'];
        $pageinfo['subject']=($pageinfo['subject']=="（无标题）"&&$title!="")?$title:$pageinfo['subject'];
        if(is_array($news->nn66)&&count($news->nn66)>0){
            foreach($news->nn66 as $k=>$v){
                $pageinfo[$k]=$v;
            }
        }
        if(is_array($news->nn888)&&!empty($news->nn888)){
            $nn888 = array_unique($news->nn888);
        }

        //多线程分析分页数据
        $pageno = 0;
        $oldnr="";
        $nn888=$nn888??"";
        if ($nn888!=""&&count($nn888) > 0) {
//            if(class_exists("Thread")){
//                $newArr = array();
//                foreach ($nn888 as $k => $v){
//                    if ($news->isHasCache($v)) continue;
//                    $newArr[] = $v;
//                }
//                if (count($newArr) > 0) $threadDataArray = createThreadFx($newArr, $news);
//                unset($newArr);
//            }


            //主线程做遗漏补救
             if(is_array($nn888)) {
                 for ($i = 0; $i < count($nn888); $i++) {
                     $id = $news->t->sqlvalue("select id from `cache` where url='" . $nn888[$i] . "' message<>'')");
                     if ($id == 0 || $bug == 1) {
                         $news->seturl($nn888[$i]);                                                                          //入口
                         $message = str_replace("'", "''", $news->getnr());
                         $tempnr = str_replace("'", "''", $news->yuannr);
                         $news->t->sql("update `cache` set message='$message',tempnr='$tempnr' where url='" . $nn888[$i] . "'");
                     }
                     //==page ==一一一一连接分页文章3
                     $pageinfo['message']=lianjiewenzhang($pageinfo['message'],$message);
                     inserttomy("newnr",$message);
                     $pageinfo['cur_url']=$nn888[$i];
                     debugger($pageinfo);
                 }
             }elseif(is_string($nn888)) {
                 $id = $news->t->sqlvalue("select id from `cache` where url='" . $nn888 . "' message<>'')");
                 if ($id == 0 || $bug == 1) {
                     $news->seturl($nn888);
                     $message = str_replace("'", "''", $news->getnr());
                     $tempnr = str_replace("'", "''", $news->yuannr);
                     $news->t->sql("update `cache` set message='$message',tempnr='$tempnr' where url='" . $nn888 . "'");
                 }
                 $pageinfo['message']=lianjiewenzhang($pageinfo['message'],$message);
                 inserttomy("newnr",$message);
                 $pageinfo['cur_url']=$nn888;
                 debugger($pageinfo);
             }
            //$message_fy=$news->getMessageArray($nn888);
            //$pageinfo['message'].=$message_fy;
        } else {

            //yzs //内容 ★★★★正文内容分页开始 ==nextpage
            if (!$news->nopage && !$news->secexe && $gather_type != 2&& $news->maxPageCount>1||$news->nextPage!="") {
                if ($news->nextPage == "") {
                    if ($news->nn88!=""&&count($news->nn88) > 0) {
                        $nextpage = getNextPage($news->nn88, $news->yuannr);
                    } else {
                        if(is_string($news->nextpagerem)) $nextpagerem = trim($news->nextpagerem);
                        $nextpagerez = ($news->nextpagere) ? trim($news->nextpagere) : "";
                        $nextpagerez .= (g(trim($news->nextpagere))) ? g(trim($news->nextpagere)) : "";
                        $g_yuannr=$news->yuannr;//.g($news->yuannr)
                        //$re = "<a[^<>]*href=(?:[\'\\\"\s]*)??(?!\#)([^<>\\\"\'\s]+?)[^<>]*>(?:<[^<>]*>|\s|\[)*(?:下一页|下一页|下一頁|下页|后一页|下一张|后页|\bNext\b|(?:(?:\&gt\;)+)(?=\<|\s)|\>\>" . $nextpagerez . ")";
                        $re = "<a[^<>]*href=(?:\'|\"|\s)??(?!\#)([^<>\\\"\'\s]+?)[^<>]*>(?><[^<>]*>|\s|\[)*(?:下一页|下一页|下一頁|下页|后一页|下一张|后页|\bNext\b|(?:(?:(?:\&gt\;)+\s*<)+)(?=\<|\s)|\>\>" . $nextpagerez . ")";
                        preg_match_all("/" . $re . "/Uisx", $g_yuannr , $mm);
                        if(count($mm[1]) < 1){
                            preg_match_all("/" . $re . "/Uisx", g($g_yuannr) , $mm);
                        }

                        if (count($mm[1]) < 1) { //####
                            $re = "(?:href|location)=(?:[\'\\\"\s]*)??(?!\#)([^<>\"\']+?)(?>[^<>]*)(?:alt|title|class)\s*=[\'\\\"\s]?(下一页|下一页|下一頁|下页|后一页|下一张|后页|\bNext\b|\bpageNext\b)";
                            preg_match_all("/" . $re . "/Uisx",$g_yuannr, $mm);
                        }
                        if (count($mm[1]) < 1) {
                            $re = "(?:alt|title|class)=[\'\"\s]??(?!\#)(?:下一页|下一頁|下一页|下页|后一页|下一张|后页|\bNext\b|\bpageNext\b)[\'\"\s]?[^<>]*(?:href|location)\s*=[\'\\\"\s]??([^<>\'\"]+?)";
                            preg_match_all("/" . $re . "/Uisx", $g_yuannr, $mm);
                        }
                        unset($g_yuannr);unset($nextpagerez);unset($nextpagerem);
                    }
                } else {
                    $nextpage = $news->nextPage;
                }

                if($mm==null||!is_array($mm)) {
                    $mm[1]=array();
                    $mm[1][0]=array();
                }
                if ($pageno <= $news->maxPageCount && (count($mm[1]) > 0 || $nextpage != "")) {
                    jsstart("top");
                    $oldurl = "";
                    $oldooo = $oldoo = "";
                    do {
                        if (ceil(js("top")) > 40 && $pageno < 20 || ceil(js("top")) > 45) {
                            $pageinfo['topsj'] = ceil(js("top"));
                            break;
                        }
                        $nextpage = ($news->nextPage == "") ? getNextPage($news->nn88, $news->yuannr) : $news->nextPage;
                        if ($nextpage != "") {
                            $mm[1][0] = $nextpage;
                        } else {
                            if ($re == "") break;
                            preg_match_all("/" . $re . "/Uisx", $news->yuannr, $mm);
                            if (count($mm[1]) < 1 || preg_match("/javascript|\(|\)/Uisx", $mm[1][0])) break;
                            $mm[1][0] = preg_replace("/http\:\/(?!\/)/Uisx", "/", $mm[1][0]);
                        }
                        $mm[1][0] = quanurl(trim($mm[1][0]), $url, $news->baseurl); //####
                        $nextpage = $mm[1][0];
                        //yzs 下一页判断
                        if (is_root($nextpage)) break;
                        if (is_toodif($nextpage, $url)) break;
                        //在抓一篇分页分章中，如果总时间限制时间到，分页没有抓完，就执行退出
                        if (js("zhongsj") > $GLOBALS['zhongsj']) {//每篇文章最多抓30秒钟时间
                            break;
                        }

                        if ($mm[1][0] == $oldurl || $mm[1][0] == $url || preg_match("/([0-9_\-]+)??(_|\-)([0-9]+)\.htm/Uisx", $mm[1][0], $oo) && $oldurl != "" && $oo[1] != "" && $oldoo != $oo[1] && $oldooo != $oo[3]) {
                            break;
                        } else {
                            $oldurl = $mm[1][0];
                            $oldoo = $oo[1];
                            $oldooo = $oo[3];
                        }

                        //pagebug($news,$re,$mm[1][0]);
                        //$news = new Zqnews($mm[1][0]);
                        //##1 todo 准备进行分页处理
                        $GLOBALS['page'] = ($pageno == "") ? 2 : $pageno;
                        $news->seturl($mm[1][0]);
                        if ($news->isfail) continue;
                        if ($news->oldnextpage != $nextpage) {
                            $news->oldnextpage = $nextpage;
                            $newnr = $news->getnr();//入口
                        }
                        $pageinfo['bug'] = print_r($news->_re, 1);
                        //==page ==一一一一连接分页文章1
                        $pageinfo['message'] = lianjiewenzhang($pageinfo['message'], $newnr);
                        $pageinfo['cacheid'] = $news->cacheid;
                        inserttomy("newnr", $newnr);
                        $pageinfo['cur_url'] = $news->url;
                        debugger($pageinfo);
                        $pageno++;
                        //$news->nr=(!isutf8($news->nr))?u($news->nr):$news->nr;
                        if ($oldnr == $news->yuannr) {
                            break;
                        } else {
                            $oldnr = $news->yuannr;
                        }
                        if ($pageno > $news->maxPageCount || strim($newnr) == "") break;
                    } while (1);
                    unset($oldnr);
                    unset($newnr);
                    unset($oldurl);
                    unset($oldoo);
                    unset($oldooo);
                }


                if ($pageno == 0&&is_array($nextpagerem)&& count($nextpagerem) > 0) {
                    for ($i = 0; $i < count($nextpagerem); $i++) {
                        do {
                            preg_match_all("/" . trim($nextpagerem[$i]) . "/Uisx", $news->ytempnr, $mm);
                            if (count($mm[1]) < 1 || preg_match("/javascript/Uisx", $mm[1][0])) break;
                            if(is_root($mm[1][0])) break;
                            $mm[1][0] = quanurl($mm[1][0], $url, $news->baseurl);
                            if ($mm[1][0] == $oldurl) {
                                break;
                            } else {
                                $oldurl = $mm[1][0];
                            }
                            //$news = new Zqnews($mm[1][0]);
                            $news->seturl($mm[1][0]);
                            $pageinfo['bug'] = print_r($news->_re,1);
                            //==page ==一一一一连接分页文章2
                            $pageinfo['message']=lianjiewenzhang($pageinfo['message'],$news->getnr());
                            //入口
                            $pageno++;
                            if ($pageno > $news->maxPageCount || strim($newnr) == "") break;
                        } while (1 == 1);
                    }
                }
            }
        }
        $pageinfo=$this->clpageinfo($pageinfo);
        //内容 第二次入库 ，--yzs 后缓存 将pageinfo写进  ===savecache===
        $pageinfo['cacheid']=$news->getcacheidbyurl($yurl);
        $pageinfo=$news->savePageinfo($yurl,$pageinfo,"","","",array("gather_type"=>1),$news);
        if(is_string($pageinfo)){
            $pageinfo=json_decode($pageinfo,1);
        }
        $pageinfo['major']=$news->major;
        return $pageinfo;
    }

    function err($info){
        $f=$_SERVER['DOCUMENT_ROOT']."/log/error_".date("Ymd").".txt";
        createdir(dirname($f));
        $str=date("H:i:s")." gather_type:{$this->gather_type} ".$this->url." ".$info.PHP_EOL;
        file_put_contents($f,$str,FILE_APPEND);
    }

    function last($info,$arrinfo=array()){
        $f=$_SERVER['DOCUMENT_ROOT']."/log/last_".date("Ymd").".txt";
        createdir(dirname($f));
        $str=date("H:i:s")." gather_type:{$this->gather_type} ".$this->url." ".$info."\r\n";
        if(!empty($arrinfo)){
            $str.=print_r($arrinfo,1);
        }
        file_put_contents($f,$str);
    }

    function clpageinfo_end($pageinfo){
        global $source,$endline,$mllb,$viewlists,$bfid;
        if(is_string($pageinfo)) return $pageinfo;
        if ($pageinfo['laiyuan'] != "") {
            $endline .= "　　[b]来源[/b]:" . $pageinfo['laiyuan'];
        }


        if ($pageinfo['bj'] != "") {
            $endline .= "　　[b]编辑[/b]:" . $pageinfo['bj'];
        } elseif ($pageinfo['author'] != "") {
            $endline .= "　　[b]编辑[/b]:" . $pageinfo['author'];
        }

        if ($endline != "" && !$pageinfo['isBao']) {
            //$pageinfo['message'].="\r\n".$endline;
        }


        $pageinfo['dateline'] = bzhsj($pageinfo['dateline']);
        $pageinfo['viewlists'] = $viewlists;
        $pageinfo['bfid'] = $bfid;
        $pageinfo['inx'] = $mllb->firstinx;
        $pageinfo['inx']=($pageinfo['inx']==null)?"":$pageinfo['inx'];
        $pageinfo['jianjie']=($pageinfo['jianjie']==null)?"":strip_tags($pageinfo['jianjie']);
        $pageinfo['thumb']=($pageinfo['thumb']==null)?array():$pageinfo['thumb'];
        $pageinfo['bfid']=($pageinfo['bfid']==null)?"":$pageinfo['bfid'];
        $pageinfo['laiyuan']=(strip_tags($pageinfo['laiyuan'])==null)?"":strip_tags($pageinfo['laiyuan']);
        if(preg_match("/\[flash\](.*)\[\/flash\]/Uisx",$pageinfo['flash'],$nnswf)){
            $pageinfo['flash']=$nnswf[1];
            if($pageinfo['aviimage']==""&&preg_match("/aviimage=([^&]+)/i",$nnswf[1],$nn23)){
                $pageinfo['aviimage']=$nn23[1];
            }
        }
        if ($_REQUEST['source'] == "") unset($pageinfo['source']);
        if ($GLOBALS['returncookie']) {
            $pageinfo['cookie'] = $GLOBALS['returncookie'];
        }
        if($pageinfo['status']==""){
            $pageinfo['status']="ok";
        }
        return $pageinfo;
    }

    function clpageinfo($pageinfo){
        global $news,$url,$rs;
        $pageinfo['isBao']=$pageinfo['isBao']??"";
        //$news->isBao=$news->isBao??"";
        //$pageinfo['isBao']=$news->isBao;
        $pageinfo['message'] = str_replace('&#13;', '', $pageinfo['message']);
        $pageinfo['message'] = preg_replace("/【?上一页([0-9 \.]|…|【|】)+下一页】?/Uisx", "", $pageinfo['message']);
        $pageinfo['status'] = (strlen($pageinfo['message']) < 50 && strlen($pageinfo['subject']) < 4) ? 'error' : 'ok';
        $pageinfo['s_url'] = $url;
        if ($news!== null) {
            $news->rs = $news->rs?? array();
        } else {
            $news = new stdClass();
            $news->rs = array();
        }
        $rs=$news->rs;
        if(isset($rs["y"])&&$rs["y"]!= "") {
            $pageinfo['message']=utf8encode($pageinfo['message']);
            $pageinfo['subject']=utf8encode($pageinfo['subject']);
        }

        if (isset($rs["p"])&&$rs["p"] >0) {
            $pageinfo['message']=paiban($pageinfo['message']);
        }


        if (isset($rs["puhuo"])&&$rs['puhuo'] != "") {
            $cc = preg_split("/\|\|\||$/m", $rs['puhuo']);
            for ($i = 0; $i < count($cc); $i++) {
                if (preg_match("/\:([a-z_0-9]+)(?=$|\:)/Uisx", trim($cc[$i]), $mm4)) {
                    $cc[$i] = preg_replace("/\:[a-z_0-9]+(?=$|\:)/Uisx", "", trim($cc[$i]));  //捕获:
                    preg_match_all("/" . trim($cc[$i]) . "/Uisx", $news->yuannr, ${"nn" . $mm4[1]});
                    //echo trim($cc[$i])."\r\n";
                    $tempnn = ${"nn" . $mm4[1]};
                    if (trim($tempnn[1][0]) != "") {
                        $pageinfo[$mm4[1]] = unicode_decode($tempnn[1][0]);
                        $pageinfo['message'] = trim($pageinfo['message']) . "\r\n" . trim($mm4[1]) . "='" . unicode_decode0(trim($tempnn[1][0])) . "'\r\n[align=center][/align]";
                    }
                }
                if (preg_match("/\>\>([a-z_0-9]+)(?=$|\:)/Uisx", trim($cc[$i]), $mm4)) {
                    $cc[$i] = preg_replace("/\>\>[a-z_0-9]+(?=$|\:)/Uisx", "", trim($cc[$i]));
                    preg_match_all("/" . trim($cc[$i]) . "/Uisx", $news->nr, ${"nn" . $mm4[1]});
                    $tempnn = ${"nn" . $mm4[1]};
                    $arr = sjson_decode($tempnn[1][0]);
                    if (!empty($arr)) {
                        foreach ($arr as $k => $v) {
                            if (is_array($v)) {
                                $pageinfo[$k] = json_encode($v);
                            } else {
                                $pageinfo[$k] = $v;
                            }
                            $pageinfo['message'] = trim($pageinfo['message']) . "\r\n" . trim($k) . "='" . $pageinfo[$k] . "'\r\n[align=center][/align]";
                        }
                    }
                }

            }
        }
        if (!$_GET['onlyread']) {
            unset($pageinfo['bug']);
        }
        preg_match_all("/\[img\].*\[\/img\]/Uisx", $pageinfo['message'],$nn09);
        for($imm=0;$imm<count($nn09[0]);$imm++){
            preg_match_all("/".addsla($nn09[0][$imm])."/Uisx",$pageinfo['message'],$nn009);
            if(count($nn009[0])>1){
                $pageinfo['message']=preg_replace("/".addsla($nn09[0][$imm])."/Uisx","tmmmm",$pageinfo['message'],1);
                $pageinfo['message']=preg_replace("/(?:　|\s)*?".addsla($nn09[0][$imm])."(?:　|\s)?/Uisx","",$pageinfo['message']);
                $pageinfo['message']=str_replace("tmmmm",$nn09[0][$imm],$pageinfo['message']);
            }
        }
        return $pageinfo;
    }
}

function class2array($array)
{
    if (is_object($array)) {
        $array = (array)$array;
    }
    if (is_array($array)) {
        foreach ($array as $key => $value) {
            $array[$key] = class2array($value);
        }
    }
    return $array;
}

function mj2array($array)
{
    $newarr = array();
    if (!empty($array)) {
        foreach ($array as $k => $v) {
            $array[$k][key] = $k;
            $newarr[] = $array[$k];
        }
    }
    return $newarr;
}

function sjson_decode($nr)
{
    $nr = preg_replace("/^[^\{\}]*?\{(.*)\}\;?/isx", "(\\1)", $nr);/**/
    $nr = preg_replace("/\]/", ")", $nr);
    $nr = preg_replace("/\[/", "(", $nr);
    $nr = preg_replace("/\(/", "array(", $nr);
    $nr = preg_replace("/\"\:/", "\"=>", $nr);
    $nr = preg_replace("/\{/", "array(", $nr);
    $nr = preg_replace("/\}/", ")", $nr);
    eval("\$arr=" . trim(unicode_decode1($nr)) . ";");
    return $arr;
}


?>